package com.test;

import java.util.List;

import com.jfinal.kit.LogKit;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

public class GithubRepoPageProcessor implements PageProcessor {

	@Override
	public void process(Page page) {
		Selectable sel = page.getHtml().xpath("//div[@class='article-content']/*[@class='img-container']/img/@src");
		List<String> title = sel.all();
		page.putField("title", title);
		for (String s : title) {
			LogKit.info(s);
		}
	}

	@Override
	public Site getSite() {
		Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
		return site;
	}

	public static void main(String[] args) {
		Spider.create(new GithubRepoPageProcessor()).addUrl(
				"https://www.baidu.com/home/news/data/newspage?nid=9902519521900763815&n_type=0&p_from=1&dtype=-1")
				// .addPipeline(new JsonFilePipeline("F:/pic"))
				.thread(5).run();
	}
}
